import datasets
from datasets import *

#加载保存后的数据集
# process_data = datasets.load_from_disk("D:\codes\sshcode\HuggingFace\save_dataset")
# print(process_data)

#加载csv文件数据集
dataset1 = load_dataset("csv",data_files=[r"D:\datasets\new-title-chinese\train.csv",
                                              r"D:\datasets\new-title-chinese\dev.csv"])
print(dataset1)
print('\n')

dataset2 = Dataset.from_csv(r"D:\datasets\new-title-chinese\train.csv",split="train")
print(dataset2)
print('\n')

#加载文件夹下面的所有csv文件数据集
dataset3 = load_dataset("csv",data_dir=r"D:\datasets\new-title-chinese")
print(dataset3)
print('\n')

import pandas as pd
data = pd.read_csv(r"D:\datasets\new-title-chinese\train.csv")
dataset4 = Dataset.from_pandas(data)
print(dataset4)
